import json
from PIL import Image, ImageDraw
from PIL import ImageColor
import re

additional_colors = [colorname for (colorname, colorcode) in ImageColor.colormap.items()]

def plot_bounding_boxes(im, noun_phrases_and_positions):
    """
    Plots bounding boxes on an image with markers for each noun phrase, using PIL, normalized coordinates, and different colors.

    Args:
        img_path: The path to the image file.
        noun_phrases_and_positions: A list of tuples containing the noun phrases
         and their positions in normalized [y1 x1 y2 x2] format.
    """

    # Load the image
    img = im
    width, height = img.size
    print(img.size)
    # Create a drawing object
    draw = ImageDraw.Draw(img)

    # Define a list of colors
    colors = [
    'red',
    'green',
    'blue',
    ] + additional_colors

    # Iterate over the noun phrases and their positions
    for i, (noun_phrase, (y1, x1, y2, x2)) in enumerate(
        noun_phrases_and_positions):
        # Select a color from the list
        color = colors[i % len(colors)]

        # Convert normalized coordinates to absolute coordinates
        abs_x1 = int(x1/1000 * width)
        abs_y1 = int(y1/1000 * height)
        abs_x2 = int(x2/1000 * width)
        abs_y2 = int(y2/1000 * height)

        # Draw the bounding box
        draw.rectangle(
            ((abs_x1, abs_y1), (abs_x2, abs_y2)), outline=color, width=4
        )

        # Draw the text
        draw.text((abs_x1 + 8, abs_y1 + 6), noun_phrase, fill=color)

    # Display the image
    img.show()

# @title Parsing utils
def parse_list_boxes(text):
  result = []
  for line in text.strip().splitlines():
    # Extract the numbers from the line, remove brackets and split by comma
    try:
      numbers = line.split('[')[1].split(']')[0].split(',')
    except:
      numbers =  line.split('- ')[1].split(',')

    # Convert the numbers to integers and append to the result
    result.append([int(num.strip()) for num in numbers])

  return result



def parse_json_in_output(output):
    """
    Extracts and converts JSON-like data from the given text output to a Python dictionary.
    
    Args:
        output (str): The text output containing the JSON data.
    
    Returns:
        dict: The parsed JSON data as a Python dictionary.
    """
    # Regex to extract JSON-like portion
    json_match = re.search(r"\{.*?\}", output, re.DOTALL)
    if json_match:
        json_str = json_match.group(0)
        # Fix single quotes and ensure proper JSON formatting
        json_str = json_str.replace("'", '"')  # Replace single quotes with double quotes
        try:
            # Convert the fixed JSON string into a dictionary
            json_data = json.loads(json_str)
            return json_data
        except json.JSONDecodeError:
            return "The extracted JSON is still not valid after formatting."
    else:
        return "No JSON data found in the given output."

im = "hospital_scene.jpeg"

from PIL import Image

im = Image.open("hospital_scene.jpeg")

print(im.format, im.size, im.mode)

im.show()

JPEG (1792, 1024) RGB

im = "prescription1.jpg"

from PIL import Image

im = Image.open("prescription1.jpg")

print(im.format, im.size, im.mode)

im.show()

MPO (2719, 3582) RGB

def promptLLM(prompt : str = None, sysprompt : str = None,  image : str = None, wantJson : bool = False, returnDict : bool = False):
    returnValue = ""
    messages = [{"role": "system", "content" : sysprompt}]
    modelToUse = TEXTMODEL
    #force it to be a json answer prompt
    #prompt = prompt if not wantJson else returnJSONAnswerPrompt(prompt)
    messages.append({"role": "user", "content": [{ 
        "type" : "text", 
        "text" : prompt 
    }]})
    if image is not None:
        image = f"data:image/jpeg;base64,{image}"
        messages[1]["content"].append({"type": "image_url", "image_url": { "url" : image}})
        modelToUse = IMGMODEL

    if wantJson:
        returnValue = openAIclient.chat.completions.create(
            model=modelToUse,
            #max_tokens= 400,
            response_format={ "type": "json_object" },
            messages=messages,
            temperature=0,
            #n=1,
        )
    else :
        returnValue = openAIclient.chat.completions.create(
            model=modelToUse,
            messages=messages,
            temperature=0,
            #n=1,
        )
    returnValue = returnValue.choices[0].message.content
    if returnDict:
        return json.loads(returnValue)
    return returnValue

import openai
from dotenv import load_dotenv  
import os
import base64
import json
import textwrap

# Function to encode the image
def encode_image(image_path):
  with open(image_path, "rb") as image_file:
    return base64.b64encode(image_file.read()).decode('utf-8')


load_dotenv()
#openAIclient = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
openAIclient = openai.OpenAI(api_key= os.getenv("OPENAI_API_KEY"))




TEXTMODEL = "gpt-4o-mini" 
IMGMODEL= "gpt-4o-mini" 

# Path to your image
img = "prescription1.jpg"

Python-dotenv could not parse statement starting at line 1
Python-dotenv could not parse statement starting at line 2
Python-dotenv could not parse statement starting at line 3
Python-dotenv could not parse statement starting at line 2
Python-dotenv could not parse statement starting at line 3

#basic call to gpt4 with prompt and image

completion = openAIclient.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What's in this image?"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{encode_image(img)}",
                        #"detail": "low"
                    }
                },
            ],
        }
    ],
)


# Wrap the text to a specified width

response = str(completion.choices[0].message)
print(textwrap.fill(response, width=120))

ChatCompletionMessage(content="The image contains handwritten notes, likely a prescription or medical instructions. It
includes a doctor's name (Dr. Markus Müller), a patient's name (Claudi Fischer), and details about a medication:
Ibuprofen, with a dosage of three 400 mg tablets, to be taken after meals. There is also a handwritten signature at the
bottom.", role='assistant', function_call=None, tool_calls=None, refusal=None)

output = promptLLM(prompt = "describe the image in detail",sysprompt = "you are a careful observer. the response should be in json format", image = encode_image(img), wantJson=True, returnDict=True)

output

{'description': {'content': [{'text': 'The image features a piece of paper with handwritten notes.'},
   {'text': "At the top left, the name 'Dr. Markus Müller' is written in cursive."},
   {'text': "Below that, the word 'Anamnese' is noted, indicating a medical context."},
   {'text': "Further down, the name 'Claudia Fischer' is written, followed by a date '1.4.1978 (f)', suggesting a birth date."},
   {'text': "The medication 'Ibuprofen' is listed, with a dosage of '3x 400mg' noted."},
   {'text': "The instruction '(nach dem Essen)' translates to 'after meals,' indicating when to take the medication."},
   {'text': "At the bottom right, there is a signature that appears to be 'Kepthülle.'"}],
  'layout': 'The text is organized in a vertical format, with varying line lengths and spacing, typical of handwritten notes.'}}

import openai
from dotenv import load_dotenv  
import os
import base64
import json
import textwrap

# Function to encode the image
def encode_image(image_path):
  with open(image_path, "rb") as image_file:
    return base64.b64encode(image_file.read()).decode('utf-8')


load_dotenv()
#openAIclient = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
openAIclient = openai.OpenAI(api_key= os.getenv("OPENAI_API_KEY"))




TEXTMODEL = "gpt-4o-mini" 
IMGMODEL= "gpt-4o-mini" 

# Path to your image
img = "hospital_scene.jpeg"

Python-dotenv could not parse statement starting at line 1
Python-dotenv could not parse statement starting at line 2
Python-dotenv could not parse statement starting at line 3
Python-dotenv could not parse statement starting at line 2
Python-dotenv could not parse statement starting at line 3

def promptLLM(prompt : str = None, sysprompt : str = None,  image : str = None, wantJson : bool = False, returnDict : bool = False):
    returnValue = ""
    messages = [{"role": "system", "content" : sysprompt}]
    modelToUse = TEXTMODEL
    #force it to be a json answer prompt
    #prompt = prompt if not wantJson else returnJSONAnswerPrompt(prompt)
    messages.append({"role": "user", "content": [{ 
        "type" : "text", 
        "text" : prompt 
    }]})
    if image is not None:
        image = f"data:image/jpeg;base64,{image}"
        messages[1]["content"].append({"type": "image_url", "image_url": { "url" : image}})
        modelToUse = IMGMODEL

    if wantJson:
        returnValue = openAIclient.chat.completions.create(
            model=modelToUse,
            #max_tokens= 400,
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "img_extract",
                    "schema": {
                    "type": "object",
                    "properties": {
                        "numberOfPeople": {
                        "type": "integer",
                        "description": "The total number of people in the environment",
                        "minimum": 0
                        },
                        "atmosphere": {
                        "type": "string",
                        "description": "Description of the atmosphere, e.g., calm, lively, etc."
                        },
                        "hourOfTheDay": {
                        "type": "integer",
                        "description": "The hour of the day in 24-hour format",
                        "minimum": 0,
                        "maximum": 23
                        },
                        "people": {
                        "type": "array",
                        "description": "List of people and their details",
                        "items": {
                            "type": "object",
                            "properties": {
                            "position": {
                                "type": "string",
                                "description": "Position of the person in the environment, e.g., standing, sitting, etc."
                            },
                            "age": {
                                "type": "integer",
                                "description": "Age of the person",
                                "minimum": 0
                            },
                            "activity": {
                                "type": "string",
                                "description": "Activity the person is engaged in, e.g., reading, talking, etc."
                            },
                            "gender": {
                                "type": "string",
                                "description": "Gender of the person",
                                "enum": ["male", "female", "non-binary", "other", "prefer not to say"]
                            }
                            },
                            "required": ["position", "age", "activity", "gender"]
                        }
                        }
                    },
                    "required": ["numberOfPeople", "atmosphere", "hourOfTheDay", "people"]
                    }}},
            messages=messages,
            temperature=0,
            #n=1,
        )
    else :
        returnValue = openAIclient.chat.completions.create(
            model=modelToUse,
            messages=messages,
            temperature=0,
            #n=1,
        )
    returnValue = returnValue.choices[0].message.content
    if returnDict:
        return json.loads(returnValue)
    return returnValue

output_image_analysis = promptLLM(prompt = "describe the image in detail",sysprompt = "you are a careful observer. the response should be in json format", image = encode_image(img), wantJson=True, returnDict=True)

#alert service prompt 

alert_sys_prompt = " you are an experienced doctor"
alert_prompt= """Extract from the following scene analysis give to you in json format, 
you should give the medicine to woman on the prescription: """ + str(output_image_analysis)

promptLLM(prompt = alert_prompt, sysprompt= alert_sys_prompt)

'Based on the scene analysis provided, the woman who is 80 years old and sitting is likely the patient. Given the calm and caring atmosphere, it would be appropriate to prescribe a medication that is suitable for her age and any potential health conditions she may have. \n\nHowever, without specific details about her medical history, symptoms, or conditions, I can only suggest a general approach. Common medications for elderly patients might include:\n\n1. **Multivitamins** - To support overall health.\n2. **Calcium and Vitamin D** - For bone health.\n3. **Antihypertensives** - If she has high blood pressure.\n4. **Statins** - If she has high cholesterol.\n5. **Metformin** - If she has diabetes.\n\nPlease consult with a healthcare professional for a tailored prescription based on her specific health needs.'

%matplotlib inline
import os
import google.generativeai as genai
from PIL import Image
import requests
import io

im = Image.open(img)

genai.configure(api_key=os.environ.get("GEMINI_API_KEY"))
model = genai.GenerativeModel("gemini-1.5-pro")

response = model.generate_content([
    im,
    (
        "Detect if there is a woman in the image and reuturn its coordinates as a list in the format '[ymin,xmin, ymax, xmax]'. Just output the list.\n "
    ),
])
response.resolve()
print(response.text)

Here are the bounding box coordinates of the women in the image:

- [162, 460, 932, 565]
- [347, 671, 557, 800]
- [378, 204, 594, 335]

# KOORDINATEN AUF BILD ZEICHNEN

from PIL import ImageDraw

# Funktion zum Zeichnen einer Box auf dem Bild
def draw_box(image, coordinates, outline="red", width=6):
    draw = ImageDraw.Draw(image)
    draw.rectangle(coordinates, outline=outline, width=width)
    return image

# Beispielkoordinaten
coordinates = [159, 462, 930, 562]


# Bild öffnen
image = Image.open(img)

# Box zeichnen
image_with_box = draw_box(image, coordinates)

# Bild anzeigen
image_with_box.show()

#boxes= parse_list_boxes(response.text)
#boxes = {f'phone_{i}': x for i, x in enumerate(boxes)}
#plot_bounding_boxes(im, noun_phrases_and_positions=list(boxes.items()))

import os
import platform
import socket
from platform import python_version
from datetime import datetime

print('-----------------------------------')
print(os.name.upper())
print(platform.system(), '|', platform.release())
print('Datetime:', datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print('Python Version:', python_version())
print('IP Address:', socket.gethostbyname(socket.gethostname()))
print('-----------------------------------')

-----------------------------------
POSIX
Linux | 6.5.0-1025-azure
Datetime: 2024-12-16 10:19:54
Python Version: 3.10.16
IP Address: 127.0.0.1
-----------------------------------

Vorname:	Silas
Nachname:	Häffner
Immatrikulationsnummer:	24672354
Modul:	Data Science
Prüfungsdatum / Raum / Zeit:	16.12.2024 / Raum: SF O3.54 / 8:00 – 12:30
Erlaubte Hilfsmittel:	w.MA.XX.DS.24HS (Data Science) Open Book, Eigener Computer, Internet-Zugang
Nicht erlaubt:	Nicht erlaubt ist der Einsatz beliebiger Formen von generativer KI (z.B. Copilot, ChatGPT) sowie beliebige Formen von Kommunikation oder Kollaboration mit anderen Menschen.

Kategorie	Beschreibung	Punkteverteilung
Code nicht lauffähig oder Ergebnisse nicht relevant	Der Code läuft nicht oder erfüllt nicht die Anforderungen des Aspekts (z. B. Bilder werden nicht geladen, die Textausgabe der Extraktion fehlt, Bounding Boxes werden nicht angezeigt).	0 Punkte
Code lauffähig, aber mit gravierenden Mängeln	Der Code läuft, jedoch fehlen zentrale Teile der Funktionalität eines Aspekts (z. B. unvollständige Extraktion von Bildinformationen oder Fehler bei der Definition eines Schemas).	25% der max. erreichbaren Punkte
Code lauffähig, aber mit mittleren Mängeln	Der Code läuft und liefert teilweise korrekte Ergebnisse für einen Aspekt, aber wichtige Details fehlen (z. B. ungenaue Bounding Boxes, unvollständige Integration der extrahierten Daten).	50% der max. erreichbaren Punkte
Code lauffähig, aber mit minimalen Mängeln	Der Code erfüllt die Anforderungen eines Aspekts weitgehend, aber kleinere Fehler oder Abweichungen (z. B. nicht robust Extraktionsdaten, kleinere Schemaabweichungen, Prompt zu wenig stringent formuliert -> teilweise unstabile Output)sind vorhanden.	75% der max. erreichbaren Punkte
Code lauffähig und korrekt	Der Code erfüllt die Anforderungen des Aspekts vollständig und liefert die erwarteten Ergebnisse ohne Fehler (z. B. korrekte Extraktion, vollständige Bounding Boxes, saubere Integration).	100% der max. erreichbaren Punkte

Arbeitspaket (AP) 3: Multimodal Interactions - Images¶

Persönliche Angaben (bitte ergänzen)¶

Bewertungskriterien¶

Vorbereitung (Dieser Teil wird nicht bewertet!)¶

Aufgabe (Dieser Teil wird bewertet!)¶

Hinweise zum folgenden Arbeitspaket:¶

Utils (Hilfsfunktionen):¶

Aufgabe (1): Laden und Visualisieren von Bildern / Importing notwendige Libraries¶

Aufgabe (2): Definition eines strukturierten Schemas für die Extraktion von Informationen¶

Aufgabe (3): Extraktion von visuellen Informationen mittels Prompting¶

Aufgabe (4): Verarbeitung der textuellen Ausgabe der visuellen Extraktion¶

Aufgabe (5): Integration der extrahierten Daten in neuem Prompt für weitere Identifizierung¶

Aufgabe (6): Visualisierung von Bounding Boxes¶

Aufgabe (7): Pipeline Validation¶

Jupyter notebook --footer info-- (please always provide this at the end of each notebook)¶